from multiprocessing import Pool, Manager
from lxml import etree
from queue import Queue
from tool import *
from sql import sqlseverDB
import requests,math
import traceback,logging

ms = sqlseverDB('{SQL Server}',SQL_SERVER, 'SuMaiTongPol', SQL_USER, SQL_PWD)
logger = logging.getLogger()
logger.setLevel(logging.INFO)  # 设置日志级别
log_path=f'日志文件/亚马逊CJDPASIN.log'
file_handler = logging.FileHandler(log_path, mode='a', encoding='utf-8')

# 创建控制台处理器 (Handler)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)

# 设置日志格式
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)

# 将处理器添加到 logger 对象
logger.addHandler(file_handler)
logger.addHandler(console_handler)


kw='t-shirt'
mutil_count=5

def process_shop(bid,kitems,queue,task_count):
    logging.info(f"进程启动: bid={bid}, 任务数={len(kitems)}")
    try:
        res= asyncio.run(cjAsin(bid,kitems,queue,task_count))
        logging.info(res)

    except Exception as e:
        logging.error(f'bid:{bid}, 出现错误: {e}')
        traceback.print_exc()
        time.sleep(3)

async def cjAsin(bid,kitems,ycj_queue,task_count):

    browser=await openSunBrowser(bid)
    page=await browser.newPage()
    for kitem in kitems:
        mid=kitem[0]
        suc=0
        fail=0
        x_ind=ycj_queue.qsize()+1
        asinList=[]
        
        ppp=1
        max_p=10
        while True:
            url=f'https://www.amazon.com/s?k={kw}&me={mid}&page={ppp}'

            try:
                await page.goto(url,{'timeout':0}) 
                await asyncio.sleep(1)
            except Exception as e:
                print(f'mid:{mid},出错 =>{e}')
            wait_count=0
            is_has_result=False
            while True:
                cur_tit= await page.title()
                if cur_tit=='Sorry! Something went wrong!':
                    logging.info(f'第{x_ind}个,mid:{mid},页面503')
                    break
                cot= await page.content()
                if len(cot)<20000:
                    logging.info(f'第{x_ind}个,mid:{mid},页面内容过少')
                    break
                
                el1=await page.xpath('//span[@class="a-color-state a-text-bold"]')
                el2=await page.xpath('//span[text()="No results for "]')
                el3=await page.xpath('//div[@data-asin!="" and @data-index]/@data-asin')
                if el1 or el3 or len(cot)>20000:
                    is_has_result=True
                    break
                if el2:
                    logging.info(f'第{x_ind}个,mid:{mid},kw:《{kw}》,无搜索结果退出采集')
                    break
                if wait_count>30:
                    logging.info(f'第{x_ind}个,mid:{mid},kw:《{kw}》,等待过久退出')
                    break
                wait_count+=1
                await asyncio.sleep(1)
            if not is_has_result:
                break
            
            cur_asins=etree.HTML(cot).xpath('//div[@data-asin!="" and @data-index]/@data-asin')
            if not cur_asins:
                break
            if ppp==1:
                el_max_ppp=etree.HTML(cot).xpath('//ul[@class="a-unordered-list a-horizontal s-unordered-list-accessibility"]/li[last()-1]/span[1]/a[1]/text()')
                if el_max_ppp:
                    max_p=int(el_max_ppp[0])
            asinList.extend(cur_asins)
            ppp+=1
            if ppp>max_p:
                break
                
        asinList=set(asinList)
        print(f'店铺:{mid},采集了 {len(asinList)} 个ASIN')
        dpurl=f'https://www.amazon.com/s?me={mid}'
        for asin in asinList:
            purl=f'https://www.amazon.com/dp/{asin}?th=1&psc=1'
            
            rs= ms.ExecQuery('select count(*) from AmazondpAsin where asin=?',(asin,))
            if rs[0][0]>0:
                fail+=1
            else:
                ct_itme=int(time.time())
                aff=ms.ExecNoQuery('insert into AmazondpAsin([ASIN],[Purl],[mid],[dpurl],[cjtime]) values(?,?,?,?,?)',(asin,purl,mid,dpurl,ct_itme))
                if aff>0:
                    suc+=1
                else:
                    fail+=1

        aff_cg=ms.ExecNoQuery('update AmazonShop set state=1,isuse=isuse+1 where shopid=?',(mid,))
        ycj_queue.put(1)
        latest_count=task_count-ycj_queue.qsize()
        print(f'第{x_ind}个店铺,mid:{mid},采集{ppp-1}页,共保存:{suc}个链接,更改采集状态:{aff_cg},剩余任务队列:{latest_count}/{task_count}\n')
    
    await page.close()



def main():
 
    print(f'第五步,审核通过店铺全店采集ASIN....')

    while True:
        rs_isuse=ms.ExecQuery('select min(isuse) from AmazonShop') #先获取采集最小次数
        min_use_count=rs_isuse[0][0]
        rs = ms.ExecQuery(f'select shopid from AmazonShop where state=0 and isuse={min_use_count}')
        if len(rs)>0:
            break
        else:
            aff=ms.ExecNoQuery('update AmazonShop set state=0 where state=1')
            print(f'关键词最小采集次数:{min_use_count},已经全部采集完毕,将所有 {aff} 个关键词采集状态置0')

    taskCount = len(rs)

    print(f'本次采集,待采集店铺总数: {taskCount}')

    #创建采集环境
    while True:
        all_bids=get_browser_environment()
        print('获取到',all_bids)
        if len(all_bids)>=mutil_count:
            bids=all_bids[:mutil_count]
            break
        else:
            print(mutil_count-len(all_bids))
            for _ in range(mutil_count-len(all_bids)):
                res=create_browser_environment_no_proxy(f'亚马逊采集{get_random_code()}')
                time.sleep(1)
    
    print(f'成功创建:{len(bids)}个采集环境')
    manager = Manager()
    queue = manager.Queue()

    qu_jian=math.ceil(taskCount/mutil_count)
    with Pool(processes=mutil_count) as pool:  # 根据实际情况设置并发进程数量
        results = []
        for j in range(mutil_count):
            kitems=rs[j*qu_jian:(j+1)*qu_jian]
            result = pool.apply_async(
                process_shop, args=(bids[j],kitems,queue,taskCount))
            results.append(result)

        # 监控任务执行情况
        pool.close()
        for result in results:
            result.wait()  # 等待每个进程完成，无超时限制

        # 所有任务执行完成后继续
        pool.join()



if __name__=='__main__':

    main()